{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"../..\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from prettytable import PrettyTable, ALL\n",
    "from textwrap import fill\n",
    "from msprof_analyze.advisor.interface.interface import Interface"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "profiling_path = \"YOUR PROFILING PATH\"\n",
    "interface = Interface(profiling_path=profiling_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 融合算子API识别"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "指定profiling路径后，可以自动识别其中包含的融合算子并给出对应的torch_npu api和需要修改的代码堆栈。基于给定堆栈可以快速定位到需要修改的代码段，替换torch_npu api后，能够减少pytorch侧的小算子的下发，进而提升模型训练速度。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "timeline_fusion_ops_result = interface.get_result(\"schedule\", \"timeline_fusion_ops\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "    <thead>\n",
       "        <tr>\n",
       "            <th>category</th>\n",
       "            <th>description</th>\n",
       "            <th>suggestion</th>\n",
       "        </tr>\n",
       "    </thead>\n",
       "    <tbody>\n",
       "        <tr>\n",
       "            <td>亲和API接口</td>\n",
       "            <td>目前运行环境版本为cann-8.0.0和torch-2.1.0，发现有2个api接口可以替换。</td>\n",
       "            <td>1. 请根据子表&#x27;Affinity training api&#x27;替换训练api接口</td>\n",
       "        </tr>\n",
       "    </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "+-------------+-----------------------------------------------------------------------+-----------------------------------------------------+\n",
       "|   category  |                              description                              |                      suggestion                     |\n",
       "+-------------+-----------------------------------------------------------------------+-----------------------------------------------------+\n",
       "| 亲和API接口 | 目前运行环境版本为cann-8.0.0和torch-2.1.0，发现有2个api接口可以替换。 | 1. 请根据子表'Affinity training api'替换训练api接口 |\n",
       "+-------------+-----------------------------------------------------------------------+-----------------------------------------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display_column_num = 3\n",
    "problems = timeline_fusion_ops_result.get(\"问题综述\")\n",
    "problem_table = PrettyTable(problems.get(\"headers\")[:display_column_num])\n",
    "for row in problems.get(\"data\"):\n",
    "    for i in range(len(row)):\n",
    "        row[i] = fill(str(row[i]), width=80)\n",
    "    problem_table.add_row(row[:display_column_num])\n",
    "\n",
    "display(problem_table)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "如下所示，存在亲和优化器和梯度裁剪两个可替换的torch_npu api，并给出了具体的堆栈。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "    <thead>\n",
       "        <tr>\n",
       "            <th>Affinity API</th>\n",
       "            <th>Code stacks</th>\n",
       "            <th>Stack called counts</th>\n",
       "        </tr>\n",
       "    </thead>\n",
       "    <tbody>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/usr/local/lib/python3.8/site-packages/torch/_tensor.py(1000): __iter__;<br>/opt/tiger/janus/janus/megatron/experts.py(2665): backward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply</td>\n",
       "            <td>4</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(35):<br>forward;  /usr/local/lib/python3.8/site-<br>packages/torch/autograd/function.py(539): apply;<br>/home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(90):<br>_npu_gmm;  /usr/local/lib/python3.8/site-packages/torch/_ops.py(692): __call__;<br>/home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(94):<br>npu_gmm;  /opt/tiger/janus/janus/megatron/experts.py(2667): backward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply</td>\n",
       "            <td>3</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/usr/local/lib/python3.8/site-packages/torch/_tensor.py(1000): __iter__;<br>/opt/tiger/janus/janus/megatron/experts.py(2389): native_moe_forward;<br>/opt/tiger/janus/janus/megatron/experts.py(2404): forward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;<br>/opt/tiger/janus/janus/megatron/experts.py(2330): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>2</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/usr/local/lib/python3.8/site-packages/torch/_tensor.py(1000): __iter__;<br>/opt/tiger/janus/janus/megatron/experts.py(2389): native_moe_forward;<br>/opt/tiger/janus/janus/megatron/experts.py(2442): forward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;<br>/opt/tiger/janus/janus/megatron/experts.py(2330): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>2</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/janus/janus/megatron/gate.py(312): calc_metric;<br>/opt/tiger/janus/janus/megatron/gate.py(392): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(405): forward_;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>2</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/Megatron-LM/megatron/core/tensor_parallel/layers.py(401): forward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;<br>/opt/tiger/Megatron-LM/megatron/core/tensor_parallel/layers.py(886):<br>linear_with_grad_accumulation_and_async_allreduce;  /opt/tiger/Megatron-<br>LM/megatron/core/tensor_parallel/layers.py(1345): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/attention.py(1005): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(680): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/mariana/mariana/models/layers/transformer.py(709): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/janus/janus/megatron/gate.py(307): calc_metric;<br>/opt/tiger/janus/janus/megatron/gate.py(392): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(405): forward_;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/janus/janus/megatron/gate.py(310): calc_metric;<br>/opt/tiger/janus/janus/megatron/gate.py(392): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(405): forward_;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/mariana/mariana/models/layers/embedding.py(316): generate_pos_embs;<br>/opt/tiger/mariana/mariana/models/layers/embedding.py(380): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/attention.py(782): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(680): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/mariana/mariana/models/layers/embedding.py(313): generate_pos_embs;<br>/opt/tiger/mariana/mariana/models/layers/embedding.py(380): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/attention.py(782): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(680): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/janus/janus/megatron/experts.py(2352): forward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;<br>/opt/tiger/janus/janus/megatron/experts.py(2330): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/Megatron-LM/megatron/model/fused_layer_norm.py(144): forward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;<br>/opt/tiger/Megatron-LM/megatron/model/fused_layer_norm.py(116): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(229): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;<br>/opt/tiger/Megatron-LM/megatron/core/tensor_parallel/layers.py(886):<br>linear_with_grad_accumulation_and_async_allreduce;  /opt/tiger/Megatron-<br>LM/megatron/model/language_model.py(38): parallel_lm_logits;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(534): lm_logits;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(486): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/janus/janus/megatron/experts.py(2667): backward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/janus/janus/megatron/gate.py(249): backward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(23):<br>forward;  /usr/local/lib/python3.8/site-<br>packages/torch/autograd/function.py(539): apply;<br>/home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(90):<br>_npu_gmm;  /usr/local/lib/python3.8/site-packages/torch/_ops.py(692): __call__;<br>/home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(94):<br>npu_gmm;  /opt/tiger/janus/janus/megatron/experts.py(2667): backward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/Megatron-LM/megatron/activate_offload.py(118): h2d_;<br>/opt/tiger/Megatron-LM/megatron/activate_offload.py(147): h2d;<br>/opt/tiger/Megatron-LM/megatron/model/fused_layer_norm.py(152): backward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/usr/local/lib/python3.8/site-packages/torch/_ops.py(692): __call__;<br>/home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(94):<br>npu_gmm;  /opt/tiger/janus/janus/megatron/experts.py(2667): backward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/janus/janus/megatron/experts.py(2450): native_scatter_backward;<br>/opt/tiger/janus/janus/megatron/experts.py(2453): forward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;<br>/opt/tiger/janus/janus/megatron/experts.py(2330): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/usr/local/lib/python3.8/site-packages/torch/_tensor.py(862): split;<br>/opt/tiger/janus/janus/megatron/experts.py(2462): forward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;<br>/opt/tiger/janus/janus/megatron/experts.py(2330): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/usr/local/lib/python3.8/site-packages/torch/_ops.py(692): __call__;<br>/home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(94):<br>npu_gmm;  /opt/tiger/janus/janus/megatron/experts.py(2390): native_moe_forward;<br>/opt/tiger/janus/janus/megatron/experts.py(2404): forward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;<br>/opt/tiger/janus/janus/megatron/experts.py(2330): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.optim.NpuFusedAdamW</td>\n",
       "            <td>/usr/local/lib/python3.8/site-packages/torch/_ops.py(692): __call__;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/profiler.py(631):<br>__enter__;  /usr/local/lib/python3.8/site-<br>packages/torch/optim/optimizer.py(361): wrapper;  /opt/tiger/Megatron-<br>LM/megatron/optimizer/optimizer.py(570): step;  /usr/local/lib/python3.8/site-<br>packages/torch/utils/_contextlib.py(115): decorate_context; /opt/tiger/Megatron-<br>LM/megatron/optimizer/distrib_optimizer.py(880): step;<br>/usr/local/lib/python3.8/site-packages/torch/utils/_contextlib.py(115):<br>decorate_context;  tasks/gpt2/unsup/model.py(1719): optimizer_step;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1559):<br>_train_one_step_optimize;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1817): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/usr/local/lib/python3.8/site-packages/torch/_tensor.py(1000): __iter__;<br>/opt/tiger/janus/janus/megatron/experts.py(2665): backward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply</td>\n",
       "            <td>4</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(35):<br>forward;  /usr/local/lib/python3.8/site-<br>packages/torch/autograd/function.py(539): apply;<br>/home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(90):<br>_npu_gmm;  /usr/local/lib/python3.8/site-packages/torch/_ops.py(692): __call__;<br>/home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(94):<br>npu_gmm;  /opt/tiger/janus/janus/megatron/experts.py(2667): backward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply</td>\n",
       "            <td>3</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/usr/local/lib/python3.8/site-packages/torch/_tensor.py(1000): __iter__;<br>/opt/tiger/janus/janus/megatron/experts.py(2389): native_moe_forward;<br>/opt/tiger/janus/janus/megatron/experts.py(2404): forward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;<br>/opt/tiger/janus/janus/megatron/experts.py(2330): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>2</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/usr/local/lib/python3.8/site-packages/torch/_tensor.py(1000): __iter__;<br>/opt/tiger/janus/janus/megatron/experts.py(2389): native_moe_forward;<br>/opt/tiger/janus/janus/megatron/experts.py(2442): forward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;<br>/opt/tiger/janus/janus/megatron/experts.py(2330): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>2</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/janus/janus/megatron/gate.py(312): calc_metric;<br>/opt/tiger/janus/janus/megatron/gate.py(392): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(405): forward_;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>2</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/Megatron-LM/megatron/core/tensor_parallel/layers.py(401): forward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;<br>/opt/tiger/Megatron-LM/megatron/core/tensor_parallel/layers.py(886):<br>linear_with_grad_accumulation_and_async_allreduce;  /opt/tiger/Megatron-<br>LM/megatron/core/tensor_parallel/layers.py(1345): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/attention.py(1005): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(680): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/mariana/mariana/models/layers/transformer.py(709): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/janus/janus/megatron/gate.py(307): calc_metric;<br>/opt/tiger/janus/janus/megatron/gate.py(392): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(405): forward_;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/janus/janus/megatron/gate.py(310): calc_metric;<br>/opt/tiger/janus/janus/megatron/gate.py(392): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(405): forward_;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/mariana/mariana/models/layers/embedding.py(316): generate_pos_embs;<br>/opt/tiger/mariana/mariana/models/layers/embedding.py(380): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/attention.py(782): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(680): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/mariana/mariana/models/layers/embedding.py(313): generate_pos_embs;<br>/opt/tiger/mariana/mariana/models/layers/embedding.py(380): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/attention.py(782): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(680): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/janus/janus/megatron/experts.py(2352): forward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;<br>/opt/tiger/janus/janus/megatron/experts.py(2330): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/Megatron-LM/megatron/model/fused_layer_norm.py(144): forward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;<br>/opt/tiger/Megatron-LM/megatron/model/fused_layer_norm.py(116): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(229): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;<br>/opt/tiger/Megatron-LM/megatron/core/tensor_parallel/layers.py(886):<br>linear_with_grad_accumulation_and_async_allreduce;  /opt/tiger/Megatron-<br>LM/megatron/model/language_model.py(38): parallel_lm_logits;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(534): lm_logits;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(486): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/janus/janus/megatron/experts.py(2667): backward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/janus/janus/megatron/gate.py(249): backward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(23):<br>forward;  /usr/local/lib/python3.8/site-<br>packages/torch/autograd/function.py(539): apply;<br>/home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(90):<br>_npu_gmm;  /usr/local/lib/python3.8/site-packages/torch/_ops.py(692): __call__;<br>/home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(94):<br>npu_gmm;  /opt/tiger/janus/janus/megatron/experts.py(2667): backward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/Megatron-LM/megatron/activate_offload.py(118): h2d_;<br>/opt/tiger/Megatron-LM/megatron/activate_offload.py(147): h2d;<br>/opt/tiger/Megatron-LM/megatron/model/fused_layer_norm.py(152): backward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/usr/local/lib/python3.8/site-packages/torch/_ops.py(692): __call__;<br>/home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(94):<br>npu_gmm;  /opt/tiger/janus/janus/megatron/experts.py(2667): backward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/opt/tiger/janus/janus/megatron/experts.py(2450): native_scatter_backward;<br>/opt/tiger/janus/janus/megatron/experts.py(2453): forward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;<br>/opt/tiger/janus/janus/megatron/experts.py(2330): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/usr/local/lib/python3.8/site-packages/torch/_tensor.py(862): split;<br>/opt/tiger/janus/janus/megatron/experts.py(2462): forward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;<br>/opt/tiger/janus/janus/megatron/experts.py(2330): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.npu_confusion_transpose</td>\n",
       "            <td>/usr/local/lib/python3.8/site-packages/torch/_ops.py(692): __call__;<br>/home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(94):<br>npu_gmm;  /opt/tiger/janus/janus/megatron/experts.py(2390): native_moe_forward;<br>/opt/tiger/janus/janus/megatron/experts.py(2404): forward;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;<br>/opt/tiger/janus/janus/megatron/experts.py(2330): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;<br>/opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;<br>/opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):<br>forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;<br>/opt/tiger/Megatron-LM/megatron/schedules.py(461):<br>forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):<br>_megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;<br>/opt/tiger/cruise/cruise/module/wrapper.py(28): forward;<br>/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):<br>_call_impl;  /usr/local/lib/python3.8/site-<br>packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):<br>_train_one_step_forward;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>torch_npu.optim.NpuFusedAdamW</td>\n",
       "            <td>/usr/local/lib/python3.8/site-packages/torch/_ops.py(692): __call__;<br>/usr/local/lib/python3.8/site-packages/torch/autograd/profiler.py(631):<br>__enter__;  /usr/local/lib/python3.8/site-<br>packages/torch/optim/optimizer.py(361): wrapper;  /opt/tiger/Megatron-<br>LM/megatron/optimizer/optimizer.py(570): step;  /usr/local/lib/python3.8/site-<br>packages/torch/utils/_contextlib.py(115): decorate_context;<br>/opt/tiger/Megatron-LM/megatron/optimizer/distrib_optimizer.py(880): step;<br>/usr/local/lib/python3.8/site-packages/torch/utils/_contextlib.py(115):<br>decorate_context;  tasks/gpt2/unsup/model.py(1719): optimizer_step;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1559):<br>_train_one_step_optimize;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(1817): _train_one_epoch;<br>/opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;<br>tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-<br>packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):<br>wrapper;  tasks/gpt2/unsup/model.py(1939): &lt;module&gt;</td>\n",
       "            <td>1</td>\n",
       "        </tr>\n",
       "    </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "|            Affinity API           |                                   Code stacks                                    | Stack called counts |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |     /usr/local/lib/python3.8/site-packages/torch/_tensor.py(1000): __iter__;     |          4          |\n",
       "|                                   |           /opt/tiger/janus/janus/megatron/experts.py(2665): backward;            |                     |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply   |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |     /home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(35):     |          3          |\n",
       "|                                   |                     forward;  /usr/local/lib/python3.8/site-                     |                     |\n",
       "|                                   |                 packages/torch/autograd/function.py(539): apply;                 |                     |\n",
       "|                                   |     /home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(90):     |                     |\n",
       "|                                   | _npu_gmm;  /usr/local/lib/python3.8/site-packages/torch/_ops.py(692): __call__;  |                     |\n",
       "|                                   |     /home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(94):     |                     |\n",
       "|                                   |      npu_gmm;  /opt/tiger/janus/janus/megatron/experts.py(2667): backward;       |                     |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply   |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |     /usr/local/lib/python3.8/site-packages/torch/_tensor.py(1000): __iter__;     |          2          |\n",
       "|                                   |      /opt/tiger/janus/janus/megatron/experts.py(2389): native_moe_forward;       |                     |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2404): forward;            |                     |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;  |                     |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2330): forward;            |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |        /opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;         |                     |\n",
       "|                                   |         /opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;         |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |     /usr/local/lib/python3.8/site-packages/torch/_tensor.py(1000): __iter__;     |          2          |\n",
       "|                                   |      /opt/tiger/janus/janus/megatron/experts.py(2389): native_moe_forward;       |                     |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2442): forward;            |                     |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;  |                     |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2330): forward;            |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |        /opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;         |                     |\n",
       "|                                   |         /opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;         |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |            /opt/tiger/janus/janus/megatron/gate.py(312): calc_metric;            |          2          |\n",
       "|                                   |              /opt/tiger/janus/janus/megatron/gate.py(392): forward;              |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |        /opt/tiger/janus/janus/megatron/lego_moe_layer.py(405): forward_;         |                     |\n",
       "|                                   |         /opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;         |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |  /opt/tiger/Megatron-LM/megatron/core/tensor_parallel/layers.py(401): forward;   |          1          |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;  |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/core/tensor_parallel/layers.py(886):       |                     |\n",
       "|                                   |     linear_with_grad_accumulation_and_async_allreduce;  /opt/tiger/Megatron-     |                     |\n",
       "|                                   |            LM/megatron/core/tensor_parallel/layers.py(1345): forward;            |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/attention.py(1005): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(680): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |     /opt/tiger/mariana/mariana/models/layers/transformer.py(709): _forward;      |          1          |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |            /opt/tiger/janus/janus/megatron/gate.py(307): calc_metric;            |          1          |\n",
       "|                                   |              /opt/tiger/janus/janus/megatron/gate.py(392): forward;              |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |        /opt/tiger/janus/janus/megatron/lego_moe_layer.py(405): forward_;         |                     |\n",
       "|                                   |         /opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;         |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |            /opt/tiger/janus/janus/megatron/gate.py(310): calc_metric;            |          1          |\n",
       "|                                   |              /opt/tiger/janus/janus/megatron/gate.py(392): forward;              |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |        /opt/tiger/janus/janus/megatron/lego_moe_layer.py(405): forward_;         |                     |\n",
       "|                                   |         /opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;         |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |  /opt/tiger/mariana/mariana/models/layers/embedding.py(316): generate_pos_embs;  |          1          |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/layers/embedding.py(380): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/layers/attention.py(782): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(680): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |  /opt/tiger/mariana/mariana/models/layers/embedding.py(313): generate_pos_embs;  |          1          |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/layers/embedding.py(380): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/layers/attention.py(782): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(680): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |            /opt/tiger/janus/janus/megatron/experts.py(2352): forward;            |          1          |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;  |                     |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2330): forward;            |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |        /opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;         |                     |\n",
       "|                                   |         /opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;         |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |     /opt/tiger/Megatron-LM/megatron/model/fused_layer_norm.py(144): forward;     |          1          |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;  |                     |\n",
       "|                                   |     /opt/tiger/Megatron-LM/megatron/model/fused_layer_norm.py(116): forward;     |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(229): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;  |          1          |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/core/tensor_parallel/layers.py(886):       |                     |\n",
       "|                                   |     linear_with_grad_accumulation_and_async_allreduce;  /opt/tiger/Megatron-     |                     |\n",
       "|                                   |           LM/megatron/model/language_model.py(38): parallel_lm_logits;           |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(534): lm_logits;     |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(486): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |           /opt/tiger/janus/janus/megatron/experts.py(2667): backward;            |          1          |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply   |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |             /opt/tiger/janus/janus/megatron/gate.py(249): backward;              |          1          |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply   |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |     /home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(23):     |          1          |\n",
       "|                                   |                     forward;  /usr/local/lib/python3.8/site-                     |                     |\n",
       "|                                   |                 packages/torch/autograd/function.py(539): apply;                 |                     |\n",
       "|                                   |     /home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(90):     |                     |\n",
       "|                                   | _npu_gmm;  /usr/local/lib/python3.8/site-packages/torch/_ops.py(692): __call__;  |                     |\n",
       "|                                   |     /home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(94):     |                     |\n",
       "|                                   |      npu_gmm;  /opt/tiger/janus/janus/megatron/experts.py(2667): backward;       |                     |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply   |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |         /opt/tiger/Megatron-LM/megatron/activate_offload.py(118): h2d_;          |          1          |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/activate_offload.py(147): h2d;          |                     |\n",
       "|                                   |    /opt/tiger/Megatron-LM/megatron/model/fused_layer_norm.py(152): backward;     |                     |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply   |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |       /usr/local/lib/python3.8/site-packages/torch/_ops.py(692): __call__;       |          1          |\n",
       "|                                   |     /home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(94):     |                     |\n",
       "|                                   |      npu_gmm;  /opt/tiger/janus/janus/megatron/experts.py(2667): backward;       |                     |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply   |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |    /opt/tiger/janus/janus/megatron/experts.py(2450): native_scatter_backward;    |          1          |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2453): forward;            |                     |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;  |                     |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2330): forward;            |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |        /opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;         |                     |\n",
       "|                                   |         /opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;         |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |       /usr/local/lib/python3.8/site-packages/torch/_tensor.py(862): split;       |          1          |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2462): forward;            |                     |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;  |                     |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2330): forward;            |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |        /opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;         |                     |\n",
       "|                                   |         /opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;         |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |       /usr/local/lib/python3.8/site-packages/torch/_ops.py(692): __call__;       |          1          |\n",
       "|                                   |     /home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(94):     |                     |\n",
       "|                                   | npu_gmm;  /opt/tiger/janus/janus/megatron/experts.py(2390): native_moe_forward;  |                     |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2404): forward;            |                     |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;  |                     |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2330): forward;            |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |        /opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;         |                     |\n",
       "|                                   |         /opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;         |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "|   torch_npu.optim.NpuFusedAdamW   |       /usr/local/lib/python3.8/site-packages/torch/_ops.py(692): __call__;       |          1          |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/autograd/profiler.py(631):      |                     |\n",
       "|                                   |                    __enter__;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |      packages/torch/optim/optimizer.py(361): wrapper;  /opt/tiger/Megatron-      |                     |\n",
       "|                                   |  LM/megatron/optimizer/optimizer.py(570): step;  /usr/local/lib/python3.8/site-  |                     |\n",
       "|                                   | packages/torch/utils/_contextlib.py(115): decorate_context; /opt/tiger/Megatron- |                     |\n",
       "|                                   |              LM/megatron/optimizer/distrib_optimizer.py(880): step;              |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/utils/_contextlib.py(115):      |                     |\n",
       "|                                   |       decorate_context;  tasks/gpt2/unsup/model.py(1719): optimizer_step;        |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1559):             |                     |\n",
       "|                                   |                            _train_one_step_optimize;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1817): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |     /usr/local/lib/python3.8/site-packages/torch/_tensor.py(1000): __iter__;     |          4          |\n",
       "|                                   |           /opt/tiger/janus/janus/megatron/experts.py(2665): backward;            |                     |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply   |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |     /home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(35):     |          3          |\n",
       "|                                   |                     forward;  /usr/local/lib/python3.8/site-                     |                     |\n",
       "|                                   |                 packages/torch/autograd/function.py(539): apply;                 |                     |\n",
       "|                                   |     /home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(90):     |                     |\n",
       "|                                   | _npu_gmm;  /usr/local/lib/python3.8/site-packages/torch/_ops.py(692): __call__;  |                     |\n",
       "|                                   |     /home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(94):     |                     |\n",
       "|                                   |      npu_gmm;  /opt/tiger/janus/janus/megatron/experts.py(2667): backward;       |                     |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply   |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |     /usr/local/lib/python3.8/site-packages/torch/_tensor.py(1000): __iter__;     |          2          |\n",
       "|                                   |      /opt/tiger/janus/janus/megatron/experts.py(2389): native_moe_forward;       |                     |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2404): forward;            |                     |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;  |                     |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2330): forward;            |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |        /opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;         |                     |\n",
       "|                                   |         /opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;         |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |     /usr/local/lib/python3.8/site-packages/torch/_tensor.py(1000): __iter__;     |          2          |\n",
       "|                                   |      /opt/tiger/janus/janus/megatron/experts.py(2389): native_moe_forward;       |                     |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2442): forward;            |                     |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;  |                     |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2330): forward;            |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |        /opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;         |                     |\n",
       "|                                   |         /opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;         |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |            /opt/tiger/janus/janus/megatron/gate.py(312): calc_metric;            |          2          |\n",
       "|                                   |              /opt/tiger/janus/janus/megatron/gate.py(392): forward;              |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |        /opt/tiger/janus/janus/megatron/lego_moe_layer.py(405): forward_;         |                     |\n",
       "|                                   |         /opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;         |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |  /opt/tiger/Megatron-LM/megatron/core/tensor_parallel/layers.py(401): forward;   |          1          |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;  |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/core/tensor_parallel/layers.py(886):       |                     |\n",
       "|                                   |     linear_with_grad_accumulation_and_async_allreduce;  /opt/tiger/Megatron-     |                     |\n",
       "|                                   |            LM/megatron/core/tensor_parallel/layers.py(1345): forward;            |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/attention.py(1005): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(680): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |     /opt/tiger/mariana/mariana/models/layers/transformer.py(709): _forward;      |          1          |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |            /opt/tiger/janus/janus/megatron/gate.py(307): calc_metric;            |          1          |\n",
       "|                                   |              /opt/tiger/janus/janus/megatron/gate.py(392): forward;              |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |        /opt/tiger/janus/janus/megatron/lego_moe_layer.py(405): forward_;         |                     |\n",
       "|                                   |         /opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;         |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |            /opt/tiger/janus/janus/megatron/gate.py(310): calc_metric;            |          1          |\n",
       "|                                   |              /opt/tiger/janus/janus/megatron/gate.py(392): forward;              |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |        /opt/tiger/janus/janus/megatron/lego_moe_layer.py(405): forward_;         |                     |\n",
       "|                                   |         /opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;         |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |  /opt/tiger/mariana/mariana/models/layers/embedding.py(316): generate_pos_embs;  |          1          |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/layers/embedding.py(380): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/layers/attention.py(782): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(680): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |  /opt/tiger/mariana/mariana/models/layers/embedding.py(313): generate_pos_embs;  |          1          |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/layers/embedding.py(380): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/layers/attention.py(782): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(680): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |            /opt/tiger/janus/janus/megatron/experts.py(2352): forward;            |          1          |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;  |                     |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2330): forward;            |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |        /opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;         |                     |\n",
       "|                                   |         /opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;         |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |     /opt/tiger/Megatron-LM/megatron/model/fused_layer_norm.py(144): forward;     |          1          |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;  |                     |\n",
       "|                                   |     /opt/tiger/Megatron-LM/megatron/model/fused_layer_norm.py(116): forward;     |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(229): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;  |          1          |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/core/tensor_parallel/layers.py(886):       |                     |\n",
       "|                                   |     linear_with_grad_accumulation_and_async_allreduce;  /opt/tiger/Megatron-     |                     |\n",
       "|                                   |           LM/megatron/model/language_model.py(38): parallel_lm_logits;           |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(534): lm_logits;     |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(486): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |           /opt/tiger/janus/janus/megatron/experts.py(2667): backward;            |          1          |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply   |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |             /opt/tiger/janus/janus/megatron/gate.py(249): backward;              |          1          |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply   |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |     /home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(23):     |          1          |\n",
       "|                                   |                     forward;  /usr/local/lib/python3.8/site-                     |                     |\n",
       "|                                   |                 packages/torch/autograd/function.py(539): apply;                 |                     |\n",
       "|                                   |     /home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(90):     |                     |\n",
       "|                                   | _npu_gmm;  /usr/local/lib/python3.8/site-packages/torch/_ops.py(692): __call__;  |                     |\n",
       "|                                   |     /home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(94):     |                     |\n",
       "|                                   |      npu_gmm;  /opt/tiger/janus/janus/megatron/experts.py(2667): backward;       |                     |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply   |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |         /opt/tiger/Megatron-LM/megatron/activate_offload.py(118): h2d_;          |          1          |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/activate_offload.py(147): h2d;          |                     |\n",
       "|                                   |    /opt/tiger/Megatron-LM/megatron/model/fused_layer_norm.py(152): backward;     |                     |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply   |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |       /usr/local/lib/python3.8/site-packages/torch/_ops.py(692): __call__;       |          1          |\n",
       "|                                   |     /home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(94):     |                     |\n",
       "|                                   |      npu_gmm;  /opt/tiger/janus/janus/megatron/experts.py(2667): backward;       |                     |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(288): apply   |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |    /opt/tiger/janus/janus/megatron/experts.py(2450): native_scatter_backward;    |          1          |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2453): forward;            |                     |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;  |                     |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2330): forward;            |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |        /opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;         |                     |\n",
       "|                                   |         /opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;         |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |       /usr/local/lib/python3.8/site-packages/torch/_tensor.py(862): split;       |          1          |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2462): forward;            |                     |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;  |                     |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2330): forward;            |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |        /opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;         |                     |\n",
       "|                                   |         /opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;         |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "| torch_npu.npu_confusion_transpose |       /usr/local/lib/python3.8/site-packages/torch/_ops.py(692): __call__;       |          1          |\n",
       "|                                   |     /home/tiger/.local/lib/python3.8/site-packages/mindspeed/ops/gmm.py(94):     |                     |\n",
       "|                                   | npu_gmm;  /opt/tiger/janus/janus/megatron/experts.py(2390): native_moe_forward;  |                     |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2404): forward;            |                     |\n",
       "|                                   |  /usr/local/lib/python3.8/site-packages/torch/autograd/function.py(539): apply;  |                     |\n",
       "|                                   |            /opt/tiger/janus/janus/megatron/experts.py(2330): forward;            |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |        /opt/tiger/janus/janus/megatron/lego_moe_layer.py(418): forward_;         |                     |\n",
       "|                                   |         /opt/tiger/janus/janus/megatron/lego_moe_layer.py(477): forward;         |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/mariana/mariana/models/layers/moe.py(468): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |     /opt/tiger/mariana/mariana/models/layers/transformer.py(735): _forward;      |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/layers/transformer.py(334): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/mariana/mariana/models/text/transformer.py(509): forward;       |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(204): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |      /opt/tiger/mariana/mariana/models/text/gpt2_megatron.py(445): forward;      |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |          /opt/tiger/Megatron-LM/megatron/model/module.py(286): forward;          |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |       /opt/tiger/Megatron-LM/megatron/model/distributed.py(230): forward;        |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |    tasks/gpt2/unsup/model.py(615): forward;  tasks/gpt2/unsup/model.py(881):     |                     |\n",
       "|                                   | forward_step;  /opt/tiger/Megatron-LM/megatron/schedules.py(305): forward_step;  |                     |\n",
       "|                                   |                /opt/tiger/Megatron-LM/megatron/schedules.py(461):                |                     |\n",
       "|                                   |         forward_backward_no_pipelining;  tasks/gpt2/unsup/model.py(961):         |                     |\n",
       "|                                   |   _megatron_fwd_bwd_function;  tasks/gpt2/unsup/model.py(1246): training_step;   |                     |\n",
       "|                                   |             /opt/tiger/cruise/cruise/module/wrapper.py(28): forward;             |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py(1527):     |                     |\n",
       "|                                   |                   _call_impl;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |          packages/torch/nn/modules/module.py(1518): _wrapped_call_impl;          |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1451):             |                     |\n",
       "|                                   |                             _train_one_step_forward;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1771): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+\n",
       "|   torch_npu.optim.NpuFusedAdamW   |       /usr/local/lib/python3.8/site-packages/torch/_ops.py(692): __call__;       |          1          |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/autograd/profiler.py(631):      |                     |\n",
       "|                                   |                    __enter__;  /usr/local/lib/python3.8/site-                    |                     |\n",
       "|                                   |      packages/torch/optim/optimizer.py(361): wrapper;  /opt/tiger/Megatron-      |                     |\n",
       "|                                   |  LM/megatron/optimizer/optimizer.py(570): step;  /usr/local/lib/python3.8/site-  |                     |\n",
       "|                                   |           packages/torch/utils/_contextlib.py(115): decorate_context;            |                     |\n",
       "|                                   |    /opt/tiger/Megatron-LM/megatron/optimizer/distrib_optimizer.py(880): step;    |                     |\n",
       "|                                   |     /usr/local/lib/python3.8/site-packages/torch/utils/_contextlib.py(115):      |                     |\n",
       "|                                   |       decorate_context;  tasks/gpt2/unsup/model.py(1719): optimizer_step;        |                     |\n",
       "|                                   |            /opt/tiger/cruise/cruise/trainer/common_trainer.py(1559):             |                     |\n",
       "|                                   |                            _train_one_step_optimize;                             |                     |\n",
       "|                                   |   /opt/tiger/cruise/cruise/trainer/common_trainer.py(1817): _train_one_epoch;    |                     |\n",
       "|                                   |          /opt/tiger/cruise/cruise/trainer/common_trainer.py(920): fit;           |                     |\n",
       "|                                   |      tasks/gpt2/unsup/model.py(1935): main;  /usr/local/lib/python3.8/site-      |                     |\n",
       "|                                   |   packages/torch/distributed/elastic/multiprocessing/errors/__init__.py(346):    |                     |\n",
       "|                                   |               wrapper;  tasks/gpt2/unsup/model.py(1939): <module>                |                     |\n",
       "+-----------------------------------+----------------------------------------------------------------------------------+---------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fusion_ops_api = timeline_fusion_ops_result.get(\"亲和API接口\")\n",
    "if fusion_ops_api:\n",
    "    fusion_ops_api_table = PrettyTable(fusion_ops_api.get(\"headers\"))\n",
    "\n",
    "    for row in fusion_ops_api.get(\"data\"):\n",
    "        for i in range(len(row)):\n",
    "            row[i] = fill(str(row[i]), width=80)\n",
    "        fusion_ops_api_table.add_row(row)\n",
    "\n",
    "    fusion_ops_api_table.hrules = ALL\n",
    "    display(fusion_ops_api_table)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
